R Notebook

This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.

Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Cmd+Shift+Enter.

library(rjson)
library(dplyr)

Attaching package: ‘dplyr’

The following objects are masked from ‘package:stats’:

    filter, lag

The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union

library(tokenizers)
library(stopwords)
library(stringr)

jobs_2020 = fromJSON(file = "indeed_job_descs_2020_09_20.json")
jobs_2021 = fromJSON(file = "indeed_job_descs_2021_01_25.json")

Getting the data into a workable DF

jobs_df = data.frame("Job", "State", "Employment", 
                     "description", 2020)

names(jobs_df) = c("Job","State","Employment", "Description", "Year")

for(i in 1:length(jobs_2020)) {
  job = jobs_2020[[i]]$request_params[1]
  state = jobs_2020[[i]]$request_params[2]
  employment = jobs_2020[[i]]$request_params[3]
  year = 2020
  
  #print(i)
  
  for(j in 1:length(jobs_2020[[i]]$job_descriptions)) {
    descript = jobs_2020[[i]]$job_descriptions[j]
    temp_df = as.data.frame(c(job,state,employment, descript, year))
    names(temp_df) = c("Job","State","Employment", "Description", "Year")
    jobs_df = rbind(jobs_df, temp_df)
    #print(j)
  }
}


head(jobs_df)

dim(jobs_df)

[1] 591   5

#unique(factor(jobs_df$State))

There is no data for anything past the 4th description

#1:4 because every job after the 4th one has no description
for(i in 1:4) {
  job = jobs_2021[[i]]$request_params[1]
  state = jobs_2021[[i]]$request_params[2]
  employment = jobs_2021[[i]]$request_params[3]
  year = 2021
  
  print(i)
  
  for(j in 1:length(jobs_2021[[i]]$job_descriptions)) {
    descript = jobs_2021[[i]]$job_descriptions[j]
    temp_df = as.data.frame(c(job,state,employment, descript, year))
    names(temp_df) = c("Job","State","Employment", "Description", "Year")
    jobs_df = rbind(jobs_df, temp_df)
    print(j)
  }
}

[1] 1
[1] 1
[1] 2
[1] 3
[1] 4
[1] 5
[1] 6
[1] 7
[1] 8
[1] 9
[1] 10
[1] 11
[1] 12
[1] 13
[1] 14
[1] 15
[1] 16
[1] 17
[1] 18
[1] 19
[1] 20
[1] 21
[1] 22
[1] 23
[1] 24
[1] 25
[1] 26
[1] 27
[1] 28
[1] 29
[1] 30
[1] 31
[1] 32
[1] 33
[1] 34
[1] 35
[1] 36
[1] 37
[1] 38
[1] 39
[1] 40
[1] 41
[1] 42
[1] 43
[1] 2
[1] 1
[1] 2
[1] 3
[1] 4
[1] 5
[1] 6
[1] 7
[1] 8
[1] 9
[1] 10
[1] 11
[1] 12
[1] 13
[1] 14
[1] 15
[1] 16
[1] 17
[1] 18
[1] 19
[1] 20
[1] 21
[1] 22
[1] 23
[1] 24
[1] 25
[1] 26
[1] 27
[1] 28
[1] 29
[1] 30
[1] 31
[1] 32
[1] 33
[1] 34
[1] 35
[1] 36
[1] 37
[1] 38
[1] 39
[1] 40
[1] 41
[1] 42
[1] 43
[1] 3
[1] 1
[1] 2
[1] 3
[1] 4
[1] 5
[1] 6
[1] 7
[1] 8
[1] 9
[1] 10
[1] 11
[1] 12
[1] 13
[1] 14
[1] 15
[1] 16
[1] 17
[1] 18
[1] 19
[1] 20
[1] 21
[1] 22
[1] 23
[1] 24
[1] 25
[1] 26
[1] 27
[1] 28
[1] 29
[1] 30
[1] 31
[1] 32
[1] 33
[1] 34
[1] 35
[1] 36
[1] 37
[1] 38
[1] 39
[1] 40
[1] 41
[1] 4
[1] 1
[1] 2
[1] 3
[1] 4
[1] 5
[1] 6
[1] 7
[1] 8
[1] 9
[1] 10
[1] 11
[1] 12
[1] 13
[1] 14
[1] 15
[1] 16
[1] 17
[1] 18
[1] 19
[1] 20
[1] 21
[1] 22
[1] 23
[1] 24
[1] 25
[1] 26
[1] 27
[1] 28
[1] 29
[1] 30
[1] 31
[1] 32
[1] 33
[1] 34
[1] 35
[1] 36
[1] 37
[1] 38
[1] 39
[1] 40
[1] 41
[1] 42

#jobs_2021
head(jobs_df)

dim(jobs_df)

[1] 760   5

#remove the first row because it is useless
jobs_df <- jobs_df[-1,]

#make the names easier for when we split
jobs_df$Job = str_replace_all(jobs_df$Job, "\\+", "_")

#remove potential sources of weird string behavior
#in a couple of samples it was relatively common to see "a.T" as in the end of one sentence into the next
jobs_df$Description = str_replace_all(jobs_df$Description, "\\.", " ")

#just some visualization of the data 
par(mar=c(11,4,4,4))
barplot(height = table(jobs_df$Job), las = 2, cex.names = .75)

table(jobs_df$Job)

         business_analyst            data_architect            data_scientist             deep_learning human_resource_specialist 
                       39                        43                        45                        42                         8 
machine_learning_engineer                 marketing            office_manager                 recruiter                researcher 
                       30                        71                        39                        82                        40 
                    sales site_reliability_engineer        software_developer              statistician             test_engineer 
                       79                        42                        41                        27                        44 
              ux_designer 
                       87 

You will need to download tokenizers and stopwords

#splitting the DF
split_df = split(jobs_df, jobs_df$Job)

#tokenize function that takes job descriptions and tokenizes them in a new DF
#That DF gets merged back with the old one 
tokenize.df = function(df) {
  
  temp = data.frame()
  for (i in 1:length(df$Description)) {
    tokens = table(tokenize_words(df$Description[i], 
                                  stopwords = stopwords::stopwords("en")))
    #temp DF to merge later 
    temp = bind_rows(temp, tokens)
    
  }
  #setting NA to 0 because it is easier to use in this case
  temp[is.na(temp)] = 0
  df = bind_cols(df,temp)
  return(df)
}

tokenize_ngrams.df = function(df) {
  
  temp = data.frame()
  for (i in 1:length(df$Description)) {
    tokens = table(tokenize_ngrams(df$Description[i], n = 3, n_min = 1,
                                  stopwords = stopwords::stopwords("en")))
    #temp DF to merge later 
    temp = bind_rows(temp, tokens)
    
  }
  #setting NA to 0 because it is easier to use in this case
  temp[is.na(temp)] = 0
  df = bind_cols(df,temp)
  return(df)
}

This part takes some time

#for loop to go through each job type and tokenize the description
for (i in 1:length(split_df)) {
  temp = split_df[i]
  split_df[[i]] = tokenize.df(temp[[1]])
  print(i)
}

[1] 1
[1] 2
[1] 3
[1] 4
[1] 5
[1] 6
[1] 7
[1] 8
[1] 9
[1] 10
[1] 11
[1] 12
[1] 13
[1] 14
[1] 15
[1] 16

This part is grabbing the totals of each column

master_li = list()
for (i in 1:length(split_df)) {
  
  #getting a temp holder
  temp = apply(split_df[[i]][,6:length(split_df[[i]])], 2, sum)
  #getting a normalization factor 
  factor = mean(apply(split_df[[i]][,6:length(split_df[[i]])], 1, sum))
  #getting some sort of normalized data for the master list
  master_li[[names(split_df)[i]]] = temp / factor
  
}

max(master_li$human_resource_specialist)

[1] 0.1767956

tokenize_resume = function(path) {
  #inputting sarah resume 
  resume = readLines(path, warn = F)
  resume = str_c(resume, sep = " ", collapse = " ") 
  
  #tokenizing the resume 
  tokens = table(tokenize_words(resume, 
                                    stopwords = stopwords::stopwords("en")))
  
  tokens = as.list(tokens, all.names = F)
  
  tokens = as.data.frame(tokens)
  return(tokens)
}
zac = tokenize_resume("test.txt")
zac

sarah = tokenize_resume("sarahresumetxt.txt")
sarah


zac = tokenize_resume("test.txt")
zac

bm = best_matches_overview(master_li, sarah)
print(sort(bm[,1], decreasing = T))

                marketing               ux_designer                 recruiter                     sales                researcher 
                25.437447                 23.099432                 23.007541                 20.967286                 14.029738 
           data_scientist             deep_learning          business_analyst             test_engineer        software_developer 
                13.443155                 13.202629                 13.018318                 12.981715                 12.911496 
           data_architect site_reliability_engineer            office_manager machine_learning_engineer              statistician 
                12.740836                 10.449200                 10.366505                  8.560117                  7.887578 
human_resource_specialist 
                 2.495229 

#converting job names to a matrix
job_names  = names(sort(bm[,1], decreasing = T)[1:3])
job_names = str_split_fixed(job_names, "", 1)

Since it is a human in the model algo we can then either determine if these seem like they will fit or not. We could just grab the first 3 and get a few more examples

best_matches = function(df, job_name, resume) {
  matches = list()
  #outerloop for the overall job category
  for (i in 1:nrow(job_name)) {
    #initialize a score and an index to return later on 
    score = 0
    index = 1
    print(job_name[i,1])
    #innerloop to get the best job match from the category
    for(j in 1:nrow(df[[i]])) {
      #temp is a DF of just the jth row only containing columns with tokenization
      temp = df[[job_name[i,1] ]][j, 6:ncol(df[[job_name[i,1]]])]
      tester = bind_rows(temp, resume)
      tester[is.na(tester)] = 0
      new_score = sum(apply(tester, 2, multi), na.rm = T)
      #changes the best seen score 
      if (new_score > score) {
        index = j 
        score = new_score 
      }
      
    }
    #sets the list with name of overall job and appends the best scoring description 
    matches[[job_name[i,1]]] = df[[job_name[i,1]]][index, 4]
    }
  return(matches)
}


best = best_matches(split_df, job_names, sarah)

[1] "marketing"
[1] "ux_designer"
[1] "recruiter"

best

$marketing
[1] "QualificationsMaster's (Required)marketing and/or communications: 8 years (Required)Full Job DescriptionPosition SummaryReporting to the Dean, the Chief Marketing and Communications Officer sets strategy and directs execution for marketing, public relations, and communications goals for all programs and departments of Columbia Business School  The Chief Marketing and Communications Officer oversees the areas of Marketing, Communications and Public Relations, and is responsible for all internal and external communications for the School, including digital technology and social media as appropriate The candidate has responsibility for the School’s branding strategy and has oversight on implementing that brand identity throughout the School including on the website, print media, marketing collateral, and all communications to internal and external stakeholders  The Chief Marketing and Communications Officer will manage a team of officers of administration ResponsibilitiesThe Chief Marketing and Communications Officer will plan and direct a coordinated communications, public relations and marketing strategy in support of Columbia Business School priorities: bridging academic theory and real-world practice; providing a world-class education that supports a lifetime career of business leadership and entrepreneurial thinking; and supporting the School’s community of students, faculty, staff and alumni Working with the dean and School senior staff, the Chief Marketing and Communications Officer will drive efforts to continue to distinguish and enhance the brand of the institution through all communications to the public, and manage media, print and web communications; direct information through media organizations and constituencies of the School The Chief Marketing and Communications Officer will work closely with the Vice Dean of Research to create a strategy for faculty research promotion to ensure that both internal and external stakeholders are aware of faculty impact beyond what is taught in the classroom  This strategy must include the website, media outreach, School-sponsored conferences and events, marketing materials, internal magazines and postings, and social media The Chief Marketing and Communications Officer will maintain the social media presence for the dean in LinkedIn, Instagram, Twitter, and on the School’s website The Chief Marketing and Communications Officer will be responsible for the implementation of the School’s new website, ensuring that stakeholder needs are addressed and deadlines are met The Chief Marketing and Communications Officer will have direct responsibility for marketing and communications for the MBA, Executive MBA and Executive Education programs and will oversee communications for other School centers, programs and departments  Measure the performance of all activities and campaigns, assess them against goals, and adjust tactics when necessary to achieve goals  Provide ongoing reporting, develop actionable insights, test and learn new approaches  Conduct thorough analyses on marketing ROI and successfully develop cost effective solutions The Chief Marketing and Communications Officer will develop strategy to increase visibility and frequency of positive media interaction of faculty, the dean and alumni of the School  Promote the School’s reputation and influence among peer business schools, prospective students and their families, alumni, and the mediaThe Chief Marketing and Communications Officer will work with the Vice Dean for External Relations and Development and the Executive Director of Alumni and Corporate Relations to provide guidance on and support for communications to alumni and corporate partners The Chief Marketing and Communications Officer will manage the dean’s communications internally and externally  S/he must capture the dean’s voice in communications internally and externally for events including commencements, orientations, welcome and update emails to faculty, staff, students, alumni, and Board members The Chief Marketing and Communications Officer will work with the Executive Director of Public Relations and oversee tactical aspects of the School’s PR program, including third-party rankings, collaborations with business media, outreach metrics and branded merchandise The Chief Marketing and Communications Officer will advise the dean’s Senior Staff in matters of internal and crisis communications with students, faculty and staff and be the liaison with the University on a coordinated messageS/he will evaluate and prioritize the print and publication program of the School as well as hold an advisory role with the CBS Publishing imprint Minimum QualificationsBachelor’s degree required  Advanced degree strongly preferred  Minimum of 8–10 years’ experience in progressively responsible roles in marketing and/or communications required, including at least four years in marketing Must have demonstrated excellent written, oral, interpersonal communication, organizational and follow-through skills  Writing and editing experience necessary, as well as managerial experience in supervising projects and/or directing the activities of communications and marketing staffs  Must have the ability to lead and work in a high volume, high demand, and highly collaborative, supportive, team oriented office  Must be organized, creative, enterprising, persuasive, and tactful  Must demonstrate flexibility and the ability to perform well under stress  Must possess understanding of the job at the strategic as well as tactical levels  Must be able to work well with people at all levels of the organization  Some evening and weekend hours required Equal Opportunity Employer / Disability / VeteranColumbia University is committed to the hiring of qualified local residents Job Type: Full-timePay: $200,000 00 - $230,000 00 per yearBenefits:401(k)401(k) matchingDental insuranceDisability insuranceHealth insurancePaid time offTuition reimbursementVision insuranceSchedule:8 hour shiftEducation:Master's (Required)Experience:marketing and/or communications: 8 years (Required)Work Remotely:Temporarily due to COVID-19"

$ux_designer
[1] "Senior UX Architect, New York\nYour Team responsibilities\nWe are searching for an experienced UX professional to join the global team developing MSCI’s next generation financial platforms  Our teams are based in US, Europe and Asia  This position, located in New York, is part of the MSCI UX team and will report to the UX Lead in New York \nYou will be responsible for building strong relationships across Design, Product, Engineering, Marketing, and Research, and create an engaging, seamless, effective user experience  In this role, we expect an analytical and creative UX professional who is able to grasp user needs and solve problems based on Lean UX Methodology \nWhat we will offer you: Depending on your location of your role, you can expect …\nCompetitive fixed and variable compensation, holiday/vacation allowance & retirement savings plans/pensions\nEmployee Resource Groups to support you in and out of the office\nA wide range of benefits including – healthcare, dental plans, risk insurances and (location dependent) - cycletowork schemes, gym benefits, retail discounts\nA purposeful approach to Wellbeing including training, support networks, membership to wellness platforms and vendors, and active local office communities\nA specific and deliberate planning to the physical offices in which we work, and support for everyone spending periods of time working remotely or at home  This approach mirrors our commitment to transparency and sustainability and puts the safety and wellness of our employees at the center of all we do  We aim to provide productive and sustainable work environments and technology that encourages collaboration, creativity and innovation \nYour key responsibilities\nUnderstand product and business requirements for both financial analytics functionality and the platform technology toolkit\nCreate user-centered design, wireframes, user flows, visual design comps and prototype using Sketch and InVision\nBuilding, iterating on and maintaining a design language system across our core products\nPresentation to internal and external users to conduct concept validation, usability testing and gather feedback\nSupport product and development teams to maintain design standards in the development process\nConduct design QA for implemented screens\nYour skills and experience that will help you excel\n7+ years of professional experience with Interaction Design for software, web applications\nPortfolio of UX-focused work samples for web and mobile applications\nBachelor’s Degree in Design or a related field\nAnalytical, detail-oriented, and problem-solving aptitude\nStrong work ethic and motivation to constantly improve the product and user experience\nInterest and ability to learn relevant financial analytics and technical subject matter\nStrong teamwork and communication skills – we actively collaborate with everyone\nProficiency in Sketch, Adobe CC, InVision and associated design tools\nDesign language, style guide, building and maintaining cohesive design system\nCan balance competing priorities, including short vs long term value and business vs user\nFamiliarity with various UX research tools, Agile and Lean UX methodologies\nPreferred Qualifications\n3+ years of experience in financial industry\nExperience managing, leading and mentoring UI, UX designers\nExcellent communication, presentation, interpersonal skills\nWork related to financial data and analytics products, trading systems, or similar\nUX work on highly technical or complex products in any field\nData visualization design experience\nExperience with HTML/CSS; JavaScript\nAccessibility\nDegree in Psychology, Social Science, Computer Science, Human Computer\nInteraction or a related field\n\n\nHow we’ll support you\nCoaching and support from experts in your team\nA performance and growth-oriented culture and values\nOpportunities for continuous learning to aid progression\nGoal based objectives and development plans\nTransparent performance-based compensation schemes\nEmployee resource groups such as the Women’s Leadership Forum, MSCIPRIDE, and Eco-Groups \nAbout MSCI and our teams:\nMSCI is a market leader in Global Indexes, Smart Beta, ESG and Risk Management, and is at the forefront of the secular trends dominating the financial services landscape today  We are committed to the future sustainability and transparency of the financial markets  We create innovative products and services that allow our clients to make more informed investment decisions, and we provide investors with critical performance measurement and risk management data and analytics  Click here to see what we do (link to MSCI brochure)\nOur values define the working environment we strive to create  We are inclusive, we champion bold ideas, we always pursue excellence, and always act with integrity  Personal accountability and responsibility are key to success, and we always work as a team to remain client centric \nMSCI is committed to developing a culture and workforce that reflects the clients and communities in which we operate  Increasing our diversity expands our talent pool which helps to accelerate innovation in all we do  We especially encourage members of historically underrepresented groups to apply, including women, ethnic minorities and those in the LGBTQ community \nTo all recruitment agencies: MSCI does not accept unsolicited CVs/Resumes  Please do not forward CVs/Resumes to any MSCI employee, location or website  MSCI is not responsible for any fees related to unsolicited CVs/Resumes \nMSCI Inc  is an equal opportunity employer committed to diversifying its workforce  It is the policy of the Firm to ensure equal employment opportunity without discrimination or harassment on the basis of race, color, religion, creed, age, sex, gender, gender identity, sexual orientation, national origin, citizenship, disability, marital and civil partnership/union status, pregnancy (including unlawful discrimination on the basis of a legally protected pregnancy/maternity leave), veteran status, or any other characteristic protected by law "

$recruiter
[1] "Great News! The Nation’s Top Physical Therapy Practice (that’s us!) is looking for lifetime members to join our unconventionally passionate family \n\nAbout the Company:\nSPEAR Physical Therapy is the APTA's- private practice section- National Practice of the Year! SPEAR Physical Therapy is one of the fastest-growing and most progressive physical therapy company in New York City  We consistently receive 5-star Yelp! reviews for our customer service, due in large part to our core team of talented and dedicated staff members  Our company values (or our SPEAR-IT Standards) are Service, Passion, Empathy, Accountability, Respect, Integrity, and Teamwork  The people we take on board live and breathe our values every day, which helps SPEAR impact the lives of over 150,000 New Yorkers a year  We have recently been named the “Nations Top Physical Therapy Company” by the Private Practice Section of the American Physical Therapy Association  This award has never been won by another New York company and is awarded to less than  01% of all physical therapy companies in the nation  SPEAR has also been awarded the “Leadership Development Award” by the New York Physical Therapy Association for a company that exemplifies the spirit, creation and implementation of leadership development  Also, SPEAR was selected to ring the bell at the New York Stock Exchange during National Small Business week \nSPEAR is also the only NYC-based privately-owned physical therapy practice accredited by New York State Department of Education to provide CEU credits to its therapists for its own proprietary on-site management and clinical training meetings (which is the fastest and most effective way to fulfill your CEU obligations)  And, its costs are fully covered by SPEAR!\nTalent Acquisition Specialist\n\nOur Shared Values - As a SPEAR team member, you naturally exhibit SPEAR’s values by default  Your talents instinctively result in actions of Service, Passion, Empathy, Accountability, Respect, Impact and Teamwork  You put people first, and know that each teammate, and each New Yorker, can rely on you to help them become the best versions of themselves \n\nYour Impact - The Talent Acquisition Specialist is responsible for the complete recruitment cycle and meeting human capital demands of SPEAR as the organization continues to grow  As the primary point of contact for all candidates, hiring managers and external vendors, you will ensure that SPEAR can efficiently staff all open roles  You have the initiative to proactively solve problems before they arise and support decisions through data reporting  The Talent Acquisition Specialist is also responsible for supporting the HR Department with onboarding, new hire orientation, system maintenance, compliance, performance management and system related items on an as needed basis \n\nOrganization Structure:\nReports to: Director, HR\nPrimary interactions: All SPEAR Employees (Including Directors) and Candidates\n\nSPEAR-IT Values:Respect & Teamwork: You approach every interaction from a place of respect and teamwork  You are committed to serving as a resource for directors and candidates  You work closely with managers and candidates so that they understand the recruitment process and the systems available to them \nImpact & Passion: You are a champion of SPEAR’s Human Resources Information System - Ceridian Dayforce and you provide reports to show the impact of Recruitment and Onboarding on the organization Empathy & Service: You provide 5-star customer service to all SPEAR employees  You respond efficiently and accordingly because you understand their feelings and needs Accountability: You take pride in meeting all recruitment deadlines and understand the impact this has on the candidate and Hiring Managers’ experience \n\nSuccess Factors:\nRecruitment:\no Respond to all inquiries and requests within 8 businesses hours \no Posts jobs externally within 24 hours of receiving notice of opening from Hiring Manager\no Connects viable candidates with Directors within 24 hours of screening\no Train all new managers on recruitment procedures and systems within 1 week of start\no Ensures strong vendor relationships with Hireright, Ceridian, Indeed, Linked In, Medbridge etc  so that any system or other issues are addressed within 24 hours \no Fosters new relationships and maintains existing relationships with career services staff at universities with DPT programs \nOnboarding:\no Issue offer letter & initiate background screening within 24 hours of a hiring decision\no Send new hires & directors Employee’s onboarding resources at least 2 business days before start date\no Assign all new hires MedBridge Compliance learning track and other role specific training 2 business days prior to employee’s start date\no Run monthly reporting to track completion of compliance training\nHR Systems\no Responds to all HR systems issues within 2 hours of reporting inclusive of\nCeridian Talent modules (Recruitment, Onboarding and Performance Management) Hireright, Medbridge, SPEAR Intranet, Indeed, Linked In,\no Complete system updates to align with department or company initiatives\n\nPosition Summary:\nRecruitment\no Monitors applications for open roles daily, identifies candidates for first round interviews\no Completes detailed documentation of a candidate’s progression through the interview process within 24 hours of an interview\no Maintains average time to fill of 30 days for all roles\no Complies and distributes weekly & monthly recruitment reports \no Ensures strong SPEAR presence at campus career fairs by working with hiring managers to solicit participation, coordinate logistics and communicate details\no Monitors and maintains Ceridian Dayforce for the Recruitment module\no Support the planning and execution of the Classroom to Clinician event twice annually\nOnboarding\no Measure Onboarding & Off boarding activities for all teams to ensure they are completed prior to New Hire start date and by Employee’s last day; disable SPEAR Intranet and Compliance training accounts\no Troubleshoots any onboarding issues and answers questions from new hires regarding offer letter, systems, and onboarding resources\no Communicates onboarding changes to all departments involved in the process\no Maintains and updates onboarding forms\nHR Systems\no Responds to all system inquiries within 4 hours; schedule training for employees within 5 business days of request\no Assigns employees to Mid-Year and Annual Reviews cycles and monitor completion of reviews; offer support as needed\nWe thank all applicants in advance for their interest in this position \nSPEAR Physical Therapy is an Equal Opportunity Employer "

hope

Error: object 'hope' not found

max(master_li$human_resource_specialist)

[1] 0.1767956

#split dataframe into new dataframes based on job description
library(dplyr)
master_df <- jobs_df %>%
  group_by(Job)

group_split(master_df)

<list_of<
  tbl_df<
    Job        : character
    State      : character
    Employment : character
    Description: character
    Year       : double
  >
>[16]>
[[1]]

[[2]]

[[3]]

[[4]]

[[5]]

[[6]]

[[7]]

[[8]]

[[9]]

[[10]]

[[11]]

[[12]]

[[13]]

[[14]]

[[15]]

[[16]]

group_keys(master_df)

#stopwords::stopwords("en", source = "snowball")

NEW METHODS:

breaaking jobs down to stem and non-stem then comparing jobs

master_li_nonfactored = list()
for (i in 1:length(split_df)) {
  
  #getting a temp holder
  temp = apply(split_df[[i]][,6:length(split_df[[i]])], 2, sum)
  #getting a normalization factor 
  #factor = mean(apply(split_df[[i]][,6:length(split_df[[i]])], 1, sum))
  #getting some sort of normalized data for the master list
  master_li_nonfactored[[names(split_df)[i]]] = temp #/ factor
  
}


#stem jobs for a new DF
stem_jobs <- c("ux_designer", "test_engineer", "site_reliability_engineer", "data_architect", "data_scientist", "software_developer", "statistician", "deep_learning", "machine_learning_engineer", "business_analyst")

#non-stem jobs for a new DF
non_stem_jobs <- c("recruiter", "marketing", "sales", "office_manager", "human_resource_specialist", "researcher")

#initializing stem DF
stem_df <- data.frame()

#creating new stem DF 
for (title in stem_jobs) {
  stem_df <- bind_rows(stem_df, master_li_nonfactored[[title]])
  
}
#removing NA and replacing with 0
stem_df[is.na(stem_df)] = 0
stem_df

#initializing 
non_stem_df <- data.frame()

#creating non stem DF
for (title in non_stem_jobs) {
  non_stem_df <- bind_rows(non_stem_df, master_li_nonfactored[[title]])
  
}
#adding 0 for NA
non_stem_df[is.na(non_stem_df)] = 0
non_stem_df

stem_tokens <- colnames(stem_df)
non_stem_tokens <- colnames(non_stem_df)

#getting unique set of tokens 
unique_stem_tokens <- setdiff(stem_tokens, non_stem_tokens)
unique_non_stem_tokens <- setdiff(non_stem_tokens, stem_tokens)

part for seeing if the resume matches stem or non stem more

stem_df = stem_df[unique_stem_tokens]
non_stem_df = non_stem_df[unique_non_stem_tokens]

#   
#total_stem = sort(apply(stem_df, 2, sum), decreasing = T)
#total_non_stem = sort(apply(non_stem_df, 2, sum), decreasing = T)


#seeing if a resume is stem or non stem 
stem_or_non = function(resume, stem, non_stem){
  count_stem = 0 
  count_non_stem = 0
  for (i in names(resume)) {
    if (i %in% names(stem)) {
      count_stem = count_stem + 1 
    }
    else if (i %in% names(non_stem)) {
      count_non_stem = count_stem + 1
    }
  }
  li = list("stem" = count_stem, "non_stem" = count_non_stem)
  return(li)
}



values = stem_or_non(zac, stem_df, non_stem_df)

lasso_mod <- glmnet(x_train, y_train, alpha=1, lambda=lasso_cv$lambda.1se)

Error in glmnet(x_train, y_train, alpha = 1, lambda = lasso_cv$lambda.1se) : 
  object 'lasso_cv' not found